﻿#region License
// Author: David Kirkland <davidrkirkland@gmail.com>
// Copyright 2007-2008 Illogikal Software 
//
// Licensed under the Apache License, Version 2.0 (the "License"); 
// you may not use this file except in compliance with the License. 
// You may obtain a copy of the License 
// 
//     http://www.apache.org/licenses/LICENSE-2.0 
//
// Unless required by applicable law or agreed to in writing, software 
// distributed under the License is distributed on an "AS IS" BASIS, 
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
// See the License for the specific language governing permissions and 
// limitations under the License.
#endregion

#region Using Directives
using System;
using System.Text;
using System.Collections.Specialized;
using System.IO;
using System.Net;
#endregion

namespace NetSpider
{
    public class HttpDocumentFetcher : IDocumentFetcher
    {


        public Document FetchFrom(string uri)
        {
            string responseString = StringGetWebPage(uri);
            //WebRequest request = HttpWebRequest.Create(uri);
            //WebResponse response = request.GetResponse();
            //Stream responseStream = response.GetResponseStream();

            //byte b = responseStream.ReadByte();

            //byte[] responseBytes = new byte[responseStream.Length];

            //responseStream.Read(responseBytes, 0, responseBytes.Length);

            //ASCIIEncoding encoding = new ASCIIEncoding();
            //string responseString = encoding.GetString(responseBytes);

            return new TextDocument(responseString);
        }


        private string StringGetWebPage(String uri)
        {
            const int bufSizeMax = 65536; // max read buffer size conserves memory
            const int bufSizeMin = 8192;  // min size prevents numerous small reads
            StringBuilder sb;




            // A WebException is thrown if HTTP request fails
            try
            {

                // Create an HttpWebRequest using WebRequest.Create (see .NET docs)!
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);




                // Execute the request and obtain the response stream
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream responseStream = response.GetResponseStream();



                // Content-Length header is not trustable, but makes a good hint.
                // Responses longer than int size will throw an exception here!
                int length = (int)response.ContentLength;



                // Use Content-Length if between bufSizeMax and bufSizeMin
                int bufSize = bufSizeMin;
                if (length > bufSize)
                    bufSize = length > bufSizeMax ? bufSizeMax : length;


                // Allocate buffer and StringBuilder for reading response
                byte[] buf = new byte[bufSize];
                sb = new StringBuilder(bufSize);






                // Read response stream until end
                while ((length = responseStream.Read(buf, 0, buf.Length)) != 0)
                    sb.Append(Encoding.UTF8.GetString(buf, 0, length));

            }
            catch (Exception ex)
            {
                sb = new StringBuilder(ex.Message);
            }
            return sb.ToString();
        }

        public virtual HttpWebRequest CreateWebRequest(string uri,
              NameValueCollection collHeader,
              string RequestMethod, bool NwCred)
        {
            HttpWebRequest webrequest =
             (HttpWebRequest)WebRequest.Create(uri);
            webrequest.KeepAlive = false;
            webrequest.Method = RequestMethod;

            int iCount = collHeader.Count;
            string key;
            string keyvalue;

            for (int i = 0; i < iCount; i++)
            {
                key = collHeader.Keys[i];
                keyvalue = collHeader[i];
                webrequest.Headers.Add(key, keyvalue);
            }

            webrequest.ContentType = "text/html";
            //"application/x-www-form-urlencoded";


            //if (ProxyServer.Length > 0)
            //{
            //    webrequest.Proxy = new
            //     WebProxy(ProxyServer, ProxyPort);
            //}
            webrequest.AllowAutoRedirect = false;

            //if (NwCred)
            //{
            //    CredentialCache wrCache =
            //            new CredentialCache();
            //    wrCache.Add(new Uri(uri), "Basic",
            //      new NetworkCredential(UserName, UserPwd));
            //    webrequest.Credentials = wrCache;
            //}
            //Remove collection elements

            collHeader.Clear();
            return webrequest;
        }

        public virtual string GetRedirectURL(HttpWebResponse
                 webresponse, ref string Cookie)
        {
            string uri = "";

            WebHeaderCollection headers = webresponse.Headers;

            if ((webresponse.StatusCode == HttpStatusCode.Found) ||
              (webresponse.StatusCode == HttpStatusCode.Redirect) ||
              (webresponse.StatusCode == HttpStatusCode.Moved) ||
              (webresponse.StatusCode == HttpStatusCode.MovedPermanently))
            {
                // Get redirected uri

                uri = headers["Location"];
                uri = uri.Trim();
            }

            //Check for any cookies

            if (headers["Set-Cookie"] != null)
            {
                Cookie = headers["Set-Cookie"];
            }
            //                string StartURI = "http:/";

            //                if (uri.Length > 0 && uri.StartsWith(StartURI)==false)

            //                {

            //                      uri = StartURI + uri;

            //                }

            return uri;
        }

        public virtual string GetFinalResponse(string ReUri,
             string Cookie, string RequestMethod, bool NwCred)
        {
            NameValueCollection collHeader =
                  new NameValueCollection();

            if (Cookie.Length > 0)
            {
                collHeader.Add("Cookie", Cookie);
            }

            HttpWebRequest webrequest =
              CreateWebRequest(ReUri, collHeader,
              RequestMethod, NwCred);

            BuildReqStream(ref webrequest);

            HttpWebResponse webresponse;

            webresponse = (HttpWebResponse)webrequest.GetResponse();

            Encoding enc = System.Text.Encoding.GetEncoding(1252);
            StreamReader loResponseStream = new
              StreamReader(webresponse.GetResponseStream(), enc);

            string Response = loResponseStream.ReadToEnd();

            loResponseStream.Close();
            webresponse.Close();

            return Response;
        }

        private void BuildReqStream(ref HttpWebRequest webrequest)
        //This method build the request stream for WebRequest
        {
            //byte[] bytes = Encoding.ASCII.GetBytes(webrequest.GetResponse().GetResponseStream().);
            //webrequest.ContentLength = bytes.Length;

            //Stream oStreamOut = webrequest.GetRequestStream();
            //oStreamOut.Write(bytes, 0, bytes.Length);
            //oStreamOut.Close();
        }

    }
}
